Advanced Lane Finding Project¶

The goals / steps of this project are the following:

Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
Apply a distortion correction to raw images.
Use color transforms, gradients, etc., to create a thresholded binary image.
Apply a perspective transform to rectify binary image ("birds-eye view").
Detect lane pixels and fit to find the lane boundary.
Determine the curvature of the lane and vehicle position with respect to center.
Warp the detected lane boundaries back onto the original image.
Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

First, I'll Compute the Camera calibration using the chessboard images.¶

import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import os

%matplotlib inline

nx = 9
ny = 6

# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((ny*nx,3), np.float32)
objp[:,:2] = np.mgrid[0:nx,0:ny].T.reshape(-1,2)

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.


# Make a list of calibration images
images = glob.glob('camera_cal/calibration*.jpg')

print(images)

# Step through the list and search for chessboard corners
for fname in images:
    img = cv2.imread(fname)
    

    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (nx,ny),None)

    # If found, add object points, image points
    if ret == True:
        
        objpoints.append(objp)
        imgpoints.append(corners)

        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, (nx,ny), corners, ret)
        plt.imshow(img)
        #cv2.waitKey(500)

cv2.destroyAllWindows()

['camera_cal/calibration7.jpg', 'camera_cal/calibration9.jpg', 'camera_cal/calibration2.jpg', 'camera_cal/calibration20.jpg', 'camera_cal/calibration5.jpg', 'camera_cal/calibration19.jpg', 'camera_cal/calibration18.jpg', 'camera_cal/calibration4.jpg', 'camera_cal/calibration1.jpg', 'camera_cal/calibration16.jpg', 'camera_cal/calibration13.jpg', 'camera_cal/calibration12.jpg', 'camera_cal/calibration6.jpg', 'camera_cal/calibration3.jpg', 'camera_cal/calibration17.jpg', 'camera_cal/calibration14.jpg', 'camera_cal/calibration10.jpg', 'camera_cal/calibration11.jpg', 'camera_cal/calibration15.jpg', 'camera_cal/calibration8.jpg']

The actual Calibration¶

img = cv2.imread(images[-1])
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

# Calibrate
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)


undst = cv2.undistort(img, mtx, dist, None, mtx)
plt.imshow(undst)

#3) Find the chessboard corners
# Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, (nx,ny),None)
# 4) If corners found: 
warped = []
if ret == True:
    print('wooooo')
else:
    print('boo')

wooooo

Perspective Transform¶

img = cv2.imread(images[-1])
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)

def corners_unwarp(img, nx, ny, mtx, dist):
    # Pass in your image into this function
    # Write code to do the following steps
    # 1) Undistort using mtx and dist
    undst = cv2.undistort(img, mtx, dist, None, mtx)
     #2) Convert to grayscale
    gray = cv2.cvtColor(undst,cv2.COLOR_BGR2GRAY)
    

    #3) Find the chessboard corners
        # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (nx,ny),None)
    # 4) If corners found: 
    warped = []
    if ret == True:
            print('wooooo')
            # a) draw corners
            # Draw and display the corners
            cv2.drawChessboardCorners(undst, (nx,ny), corners, ret)
            
            # b) define 4 source points src = np.float32([[,],[,],[,],[,]])
            #rect = order_points(corners)
            src = np.float32([corners[0], corners[nx-1], corners[-1], corners[-nx]])
                 #Note: you could pick any four of the detected corners 
                 # as long as those four corners define a rectangle
                 #One especially smart way to do this would be to use four well-chosen
                 # corners that were automatically detected during the undistortion steps
                 #We recommend using the automatic detection of corners in your code
            # c) define 4 destination points dst = np.float32([[,],[,],[,],[,]])
            
            offset = 100
            img_size = (gray.shape[1], gray.shape[0])

            dst = np.float32([[offset, offset], [img_size[0]-offset, offset], 
                                     [img_size[0]-offset, img_size[1]-offset], 
                                     [offset, img_size[1]-offset]])
            # d) use cv2.getPerspectiveTransform() to get M, the transform matrix
            m = cv2.getPerspectiveTransform(src, dst)

            # e) use cv2.warpPerspective() to warp your image to a top-down view

            warped = cv2.warpPerspective(undst, m, img_size, flags=cv2.INTER_LINEAR)
    else:
        raise NameError('Didn\'t find any chessboard corners')

    return warped, m
top_down, perspective_M = corners_unwarp(img, nx, ny, mtx, dist)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(top_down)
ax2.set_title('Undistorted and Warped Image', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

r,g,b =  cv2.split(img)
img = cv2.merge((b,g,r))
cv2.imwrite("output_images/output_original_chess.jpg", img)
r,g,b =  cv2.split(undst)
undst = cv2.merge((b,g,r))
cv2.imwrite("output_images/output_undistorted_chess.jpg", undst)
r,g,b =  cv2.split(top_down)
undst = cv2.merge((b,g,r))
cv2.imwrite("output_images/output_warp_perspective_chess.jpg", undst)

wooooo

True

Get the test road files¶

files = os.listdir("test_images/")
files

['output_test5.jpg',
 'output_perspective_test2.jpg',
 'straight_lines2.jpg',
 'output_test2.jpg',
 'output_perspective_signs_vehicles_xygrad.png',
 'output_perspective_chesstest6.jpg',
 'test3.jpg',
 'output_perspective_straight_lines1.jpg',
 'test1.jpg',
 'signs_vehicles_xygrad.png',
 'output_signs_vehicles_xygrad.png',
 'output_original_chess_test6.jpg',
 'output_straight_lines1.jpg',
 'output_perspective_straight_lines2.jpg',
 'output_test6.jpg',
 'test6.jpg',
 'straight_lines1.jpg',
 'output_perspective_test6.jpg',
 'output_perspective_test3.jpg',
 'output_perspective_test1.jpg',
 'test5.jpg',
 'test2.jpg',
 'output_undistorted_chess_test6.jpg',
 'output_perspective_test4.jpg',
 'output_test3.jpg',
 'output_perspective_test5.jpg',
 'output_test4.jpg',
 'output_straight_lines2.jpg',
 'test4.jpg',
 'output_test1.jpg']

Un-distort the Image¶

file = 'test6.jpg'
image = mpimg.imread('test_images/'+file)
# Undistort the camera image using the mtx calculated from the chess board.
undist = cv2.undistort(image, mtx, dist, None, mtx)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(undist, cmap='gray')
ax2.set_title('Undistorted Image', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)    

r,g,b =  cv2.split(undist)
result = cv2.merge((b,g,r))
cv2.imwrite('output_images/output_undistorted_'+file, result)

True

Fix Perspective¶

image = mpimg.imread('test_images/test6.jpg')
# Undistort the camera image using the mtx calculated from the chess board.
undist = cv2.undistort(image, mtx, dist, None, mtx)

def show_lines(img, lines):
    for i in range(len(lines)):
        end = (i+1) % len(lines)
        cv2.line(img, (lines[i][0], lines[i][1]), (lines[end][0], lines[end][1]), color=[0,255,0], thickness=3)

def unwarp(img, src, dst, debug=False):
    # Switch so first is x and second is y.
    img_size = (img.shape[1], img.shape[0])

    # Compute the perspective transform M
    m = cv2.getPerspectiveTransform(src, dst)
    # inverse by swapping
    m_inverse = cv2.getPerspectiveTransform(dst, src)

    
    # Warp the images
    warped = cv2.warpPerspective(img, m, img_size, flags=cv2.INTER_LINEAR)
    if(debug == True):
        show_lines(img, src)
        show_lines(warped, dst)
 
    return warped, m, m_inverse
   
SRC_TOP_Y = .65
SRC_TOP_X_LEFT = 0.444
SRC_TOP_X_RIGHT = 0.56
SRC_BOTTOM_X_LEFT = 0.281
SRC_BOTTOM_X_RIGHT = 0.880



def transform_lane_perspective(img, debug=False):
  

    img_size = (img.shape[1], img.shape[0])

    offset = 250
    
    # Select the required area using a quadrilateral.

    # Source src co-ords
    src = np.float32([(img_size[1]*SRC_BOTTOM_X_LEFT, img_size[1]), # bottom left
              (img_size[0]*SRC_TOP_X_LEFT,img_size[1]*SRC_TOP_Y), # top left
              (img_size[0]*SRC_TOP_X_RIGHT, img_size[1]*SRC_TOP_Y), # top right
             (img_size[0]*SRC_BOTTOM_X_RIGHT, img_size[1])]) # bottom right


    # Destination map coordinates
    dst = np.float32([
    [offset, img_size[1]], # bottom left
    [offset, offset], # top left
    [img_size[0]-offset, offset], # Top right
    [img_size[0]-offset, img_size[1]] ]) # Bottom Right


    unwarped, m, m_inverse = unwarp(img, src, dst, debug=debug) 
    return unwarped, m, m_inverse
    

#for file in ['straight_lines1.jpg', 'straight_lines2.jpg']:
for file in files:

    if file[0:6] != "output":
        img = mpimg.imread("test_images/" + file)
        # Undistort the camera image using the mtx calculated from the chess board.
        undist = cv2.undistort(img, mtx, dist, None, mtx)
        
        # Change to plan view.
        unwarped_lane, m, m_inverse = transform_lane_perspective(undist, debug= True)


        result = unwarped_lane

        
        # Plot the result
        f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
        f.tight_layout()
        ax1.imshow(undist)
        ax1.set_title('Original Perspective', fontsize=50)
        ax2.imshow(unwarped_lane, cmap='gray')
        ax2.set_title('Plan Perspective', fontsize=50)
        #plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
        r,g,b =  cv2.split(undist)
        result = cv2.merge((b,g,r))
        cv2.imwrite("output_images/output_unwarped_lane_rect_"+file, result)
        r,g,b =  cv2.split(unwarped_lane)
        result = cv2.merge((b,g,r))
        cv2.imwrite("output_images/output_warp_perspective_"+file, result)

Gradients and color spaces.¶

Sobel¶

# Read in an image 
image = mpimg.imread('test_images/straight_lines1.jpg')
image = mpimg.imread('test_images/test6.jpg')



# Define a function that applies Sobel x or y, 
# then takes an absolute value and applies a threshold.
# Note: calling your function with orient='x', thresh_min=20, thresh_max=100
# should produce output like the example image shown above this quiz.
def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0, 255)):
    
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the derivative in x or y given orient = 'x' or 'y'
    if(orient == 'x'):
        sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    elif orient == 'y':
        sobelx = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    else:
        raise NameError('Invalid orient: try x or y instead.')

        
    # 3) Take the absolute value of the derivative or gradient
    abs_sobelx = np.absolute(sobelx)
    # 4) Scale to 8-bit (0 - 255) then convert to type = np.uint8
    scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
    # 5) Create a mask of 1's where the scaled gradient magnitude 
            # is > thresh_min and < thresh_max
    sx_binary = np.zeros_like(scaled_sobel)
    sx_binary[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    binary_output = sx_binary
    return binary_output
    
# Run the function
grad_binary = abs_sobel_thresh(image, orient='y', thresh = (70, 100))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(grad_binary, cmap='gray')
ax2.set_title('Thresholded Gradient', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Sobel Mag¶

def mag_thresh(img, sobel_kernel=3, mag_thresh=(0, 255)):
    
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the gradient in x and y separately
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Calculate the magnitude 
    mag_sobel = np.sqrt(sobelx**2 + sobely**2)
    # 4) Scale to 8-bit (0 - 255) and convert to type = np.uint8
    scaled_sobel = np.uint8(255*mag_sobel/np.max(mag_sobel))
    # 5) Create a binary mask where mag thresholds are met
    s_binary = np.zeros_like(scaled_sobel)
    s_binary[(scaled_sobel >=  mag_thresh[0]) & (scaled_sobel <= mag_thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    binary_output = s_binary
    return binary_output
    
# Run the function
mag_binary = mag_thresh(image, sobel_kernel=3, mag_thresh=(30, 100))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(mag_binary, cmap='gray')
ax2.set_title('Thresholded Magnitude', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Sobel Direction¶

# Define a function that applies Sobel x and y, 
# then computes the direction of the gradient
# and applies a threshold.
def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
    
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the gradient in x and y separately
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Take the absolute value of the x and y gradients
    abs_sobelx = np.absolute(sobelx)
    abs_sobely = np.absolute(sobely)
    # 4) Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient 
    direction = np.arctan2(abs_sobely, abs_sobelx)
    # 5) Create a binary mask where direction thresholds are met
    s_binary = np.zeros_like(direction)
    s_binary[(direction >=  thresh[0]) & (direction <= thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    binary_output = s_binary
    return binary_output
 
    
# Run the function
dir_binary = dir_threshold(image, sobel_kernel=15, thresh=(0.7, 1.3))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(dir_binary, cmap='gray')
ax2.set_title('Thresholded Grad. Dir.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Combined Thresholds¶

# Apply each of the thresholding functions
ksize = 15

gradx        = abs_sobel_thresh(image, orient='x', sobel_kernel=ksize, thresh=(20, 100))
#gradx = grad_binary
grady = abs_sobel_thresh(image, orient='y', sobel_kernel=ksize, thresh=(20, 100))
mag_binary = mag_thresh(image, sobel_kernel=ksize, mag_thresh=(30, 100))
dir_binary = dir_threshold(image, sobel_kernel=ksize, thresh=(0.7, 1.3))

combined = np.zeros_like(dir_binary)
combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1

# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(combined, cmap='gray')
ax2.set_title('Thresholded Grad. Dir.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Color spaces¶

print('Easy Frame:')
image = mpimg.imread('test_images/test4.jpg')

thresh = (180, 255)
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
binary = np.zeros_like(gray)
binary[(gray > thresh[0]) & (gray <= thresh[1])] = 1
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(binary, cmap='gray')
ax2.set_title('Thresholded Gray.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

# VS HLS

hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
H = hls[:,:,0]
L = hls[:,:,1]
S = hls[:,:,2]


# Hue 

thresh = (20, 200)
binary = np.zeros_like(H)
binary[(H > thresh[0]) & (H <= thresh[1])] = 1

# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(H, cmap='gray')
ax1.set_title('Original Image Hue', fontsize=50)
ax2.imshow(binary, cmap='gray')
ax2.set_title('Thresholded Hue.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Easy Frame:

Color Spaces: Tricky Frame¶

print('Tricky Frame:')
image = mpimg.imread('test_images/test4.jpg')

thresh = (180, 255)
gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
binary = np.zeros_like(gray)
binary[(gray > thresh[0]) & (gray <= thresh[1])] = 1
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(binary, cmap='gray')
ax2.set_title('Thresholded Gray.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

# VS HLS

hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
H = hls[:,:,0]
L = hls[:,:,1]
S = hls[:,:,2]

# Saturation
thresh = (230, 255)

binary = np.zeros_like(S)
# X sobel
s_sob = np.absolute(cv2.Sobel(S, cv2.CV_64F, 1, 0))
s_sob = S
binary[(s_sob > thresh[0]) & (s_sob <= thresh[1])] = 1

# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(S, cmap='gray')
ax1.set_title('Original Image Saturation', fontsize=50)
ax2.imshow(binary, cmap='gray')
ax2.set_title('Thresholded Saturation.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

# Hue 
thresh = (20, 30)
binary = np.zeros_like(H)
binary[(H > thresh[0]) & (H <= thresh[1])] = 1

# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(H, cmap='gray')
ax1.set_title('Original Image Hue', fontsize=50)
ax2.imshow(binary, cmap='gray')
ax2.set_title('Thresholded Hue.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

# TODO combine red and saturation

Tricky Frame:

image = mpimg.imread('test_images/test4.jpg')

def HLS_RGB(img):
    return cv2.cvtColor(img, cv2.COLOR_HLS2RGB)
def RGB_HLS(img):
    return cv2.cvtColor(img, cv2.COLOR_RGB2HLS)

# Thresholds the S-channel of HLS
# Use exclusive lower bound (>) and inclusive upper (<=)
def hls_select(img, s_thresh=(230, 255), h_thresh=(20 , 30)):
    # 1) Convert to HLS color space
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    H = hls[:,:,0]
    L = hls[:,:,1]
    S = hls[:,:,2]

    # 2) Apply a threshold to the S channel
    s_binary = np.zeros_like(S)
    s_binary[(S > s_thresh[0]) & (S <= s_thresh[1])] = 1
    h_binary = np.zeros_like(H)
    h_binary[(H > h_thresh[0]) & (H <= h_thresh[1]) & (s_binary==1)] = 1
    
    
    return h_binary
    
# Optional TODO - tune the threshold to try to match the above image!    
hls_binary = hls_select(image, s_thresh=(230, 255))

# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(hls_binary, cmap='gray')
ax2.set_title('Thresholded S', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Combine Color and Sobel¶

def color_sob_pipeline(image,  s_thresh=(180, 255), sx_thresh=(20, 100)):
    s_binary = hls_select(image, s_thresh=(s_thresh[0], s_thresh[1]))
    
    # Apply each of the thresholding functions
    ksize = 15

    gradx        = abs_sobel_thresh(image, orient='x', sobel_kernel=ksize, thresh=(20, 90))
    #gradx = grad_binary
    grady = abs_sobel_thresh(image, orient='y', sobel_kernel=ksize, thresh=(100, 150))
    mag_binary = mag_thresh(image, sobel_kernel=ksize, mag_thresh=(30, 100))
    dir_binary = dir_threshold(image, sobel_kernel=ksize, thresh=(0, np.pi/4))

    combined = np.zeros_like(dir_binary)
    mag_dir_binary = np.zeros_like(dir_binary)
    mag_dir_binary[((mag_binary == 1) & (dir_binary == 1))] = 1
    grad_xy_binary = np.zeros_like(gradx)
    grad_xy_binary[((gradx == 1) & (grady == 1))] = 1
    combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
    
    # Combine the two approaches
    sxbinary = gradx
    s_binary = s_binary
    # Scaled so both 'blue' and 'green' are given equal value for if we due grayscale 
    
    color_binary = np.dstack((np.uint8(sxbinary), np.uint8( s_binary), np.zeros_like(sxbinary))) * 255
    both_combined = combined
    both_combined[((combined == 1) | (s_binary == 1))] = 1
    # TODO Use combined and use the params + provide full params.
    return combined

for file in files:
    if file[0:6] != "output":
        image = mpimg.imread('test_images/' + file)

        undist = cv2.undistort(image, mtx, dist, None, mtx)
        # Transform image into a top down view.
        top_down, m, m_inverse = transform_lane_perspective(undist)
        color_sob = color_sob_pipeline(top_down)
        binary = (color_sob)
        # Plot the result
        f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
        f.tight_layout()
        ax1.imshow(image)
        ax1.set_title('Original Image', fontsize=50)
        ax2.imshow(binary, cmap='gray')
        ax2.set_title('HLS S Color and sob Combined {}'.format(file), fontsize=50)
        plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.) 
        #r,g,b =  cv2.split(binary)
        #result = cv2.merge((binary,binary,binary))
        #

        plt.imsave('output_images/output_binary_filter_'+file, binary, cmap='gray')

Un-distort the camera image¶

image = mpimg.imread('test_images/test1.jpg')

# Undistort the camera image using the mtx calculated from the chess board.
undist = cv2.undistort(image, mtx, dist, None, mtx)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(undist, cmap='gray')
ax2.set_title('Undistorted Image', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Perspective transform¶

warped, m, m_inverse = transform_lane_perspective(undist)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(warped, cmap='gray')
ax2.set_title('Trapazoid for Perspective Transformation', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Histogram¶

def hist(img):
    # Grab only the bottom half of the image
    # Lane lines are likely to be mostly vertical nearest to the car
        # The proportions of image width and height used for the selected regions.

    # Sum across image pixels vertically - make sure to set `axis`
    # i.e. the highest areas of vertical lines should be larger values
    histogram = np.sum(img[img.shape[0]//2:,:], axis=0, dtype=np.float32)
  
    return histogram

def sliding_window(img):
    # Create an output image to draw on and visualize the result
    out_img = np.dstack((img, img, img))*255
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]//2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint
    
    
    
    # HYPERPARAMETERS
    # Choose the number of sliding windows
    nwindows = 9
    # Set the width of the windows +/- margin
    margin = 100
    # Set minimum number of pixels found to recenter window
    minpix = 50

    # Set height of windows - based on nwindows above and image shape
    window_height = np.int(img.shape[0]//nwindows)
    # Identify the x and y positions of all nonzero (i.e. activated) pixels in the image
    nonzero = img.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated later for each window in nwindows
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []

files = os.listdir("test_images/")
# Test on the test images.
for file in files:
    if file[0:6] != "output":
        img = mpimg.imread("test_images/" + file)
        
        # Undistort the camera image using the mtx calculated from the chess board.
        undist = cv2.undistort(img, mtx, dist, None, mtx)
        # Transform image into a top down view.
        top_down, m, m_inverse = transform_lane_perspective(undist)
        # Use sob with saturation.
        color_sob = color_sob_pipeline(top_down)
        result = color_sob
        histogram = hist(result)
        fig=plt.figure(figsize=(16, 8))
        rows = 2
        cols = 2
        fig.add_subplot(rows, cols, 1)
        plt.imshow(img) 
        fig.add_subplot(rows, cols, 2)
        plt.imshow(result, cmap='gray')
        fig.add_subplot(rows, cols, 4)
        plt.title("Histogram of Lines")
        plt.plot(histogram)
        #r,g,b =  cv2.split(hist)
        #histogram = cv2.merge((binary,binary,binary))
        #plt.imwrite("output_images/output_hist_"+file, hist)
        #plt.savefig("abc.png")
        plt.savefig('output_images/output_hist_'+file)

Line Class¶

# Define a class to receive the characteristics of each line detection
class Line():
    def __init__(self):
        # was the line detected in the last iteration?
        self.detected = False  
        # x values of the last n fits of the line
        self.recent_xfitted = [] 
        #average x values of the fitted line over the last n iterations
        self.bestx = None     
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None  
        #polynomial coefficients for the most recent fit
        self.current_left_fit = []
        #polynomial coefficients for the most recent fit
        self.current_right_fit = []
        #radius of curvature of the line in some units
        self.radius_of_curvature = None 
        #distance in meters of vehicle center from the line
        self.line_base_pos = None 
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float') 
        #x values for detected line pixels
        self.allx = None  
        #y values for detected line pixels
        self.ally = None  
        self.plot_left = []
        self.plot_right = []
        self.ploty = []

line = Line()

Measuring Curvature¶

import numpy as np


# Should see values of 533.75 and 648.16 here, if using
# the default `generate_data` function with given seed number

def measure_curvature(ploty, left_fit, right_fit):
    '''
    Calculates the curvature of polynomial functions in meters.
    '''
    # Define conversions in x and y from pixels space to meters
    ym_per_pix = (30/720)  *10# meters per pixel in y dimension
    xm_per_pix = 3.7/750 # meters per pixel in x dimension
    
    # Define y-value where we want radius of curvature
    # We'll choose the maximum y-value, corresponding to the bottom of the image
    ploty = np.linspace(0, (img.shape[0]-1), img.shape[0])
    y_eval = np.max(ploty)

    

    # Get points for warp space fits 
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    left_fitx = left_fitx[::-1]
    right_fitx = right_fitx[::-1]
    

    # fit new polynomials in world space.
    left_fit_meters = np.polyfit(ploty * ym_per_pix, left_fitx * xm_per_pix, 2)
    right_fit_meters = np.polyfit(ploty * ym_per_pix, right_fitx * xm_per_pix, 2)
    
    ## Implement the calculation of the left line here at the point y_eval
    left_curverad = ((1 + (2*left_fit_meters[0]*y_eval*ym_per_pix  + left_fit_meters[1])**2)**(3.0/2)) / np.absolute(2*left_fit_meters[0])
    
    right_curverad = ((1 + (2*right_fit_meters[0]*y_eval*ym_per_pix  + right_fit_meters[1])**2)**(3.0/2)) / np.absolute(2*right_fit_meters[0])
    

    return (left_curverad + right_curverad)/2

# Calculate
def calculate_center_diff(img):
    xm_per_pix = 3.7/750 # meters per pixel in x dimension
    if(len(line.plot_left) == 0 or len(line.plot_left) == 0):
        return -1;
    else:
        avg_chunk = 5
        if(len(line.plot_left) < 5):
            avg_chunk = len(line.plot_left)
            
        first_leftx = line.plot_left[-avg_chunk:].mean()
        first_rightx = line.plot_right[-avg_chunk:].mean()
        left_lane = first_leftx
        right_lane = first_rightx
        
        diff_left_right = (right_lane-left_lane) # diff
        center_lane = (right_lane-left_lane)//2 + left_lane # should be center lane pos
        center_car = img.shape[1]/2 # Half way across the screen pos
        diff_left_center_car = center_car - left_lane
        diff_left_center_lane = center_lane - left_lane # should be 3.7/2
        diff = center_lane - center_car
        simple_in_meters = (diff_left_center_car-diff_left_center_lane)*xm_per_pix
        # Scale to meters
#         print('dif: {} '.format(diff))
        
#         print('center lane: {}'.format(center_lane))
#         print('left lane: {}'.format(left_lane))
#         print('right lane: {}'.format(right_lane))
#         print('diff left center car: {}'.format(diff_left_center_car))
#         print('diff left center lane: {}'.format(diff_left_center_lane))
#         print('diff left right: {}'.format(diff_left_right))
#         print('center_lane_percent: {}'.format(((100.0*diff_left_center_car)/diff_left_center_lane)))
#         print('Simple Meters: {}m'.format(simple_in_meters))
#         draw_points_center_lane = (np.asarray([line.plot_left + center_lane - left_lane, line.ploty]).T).astype(np.int32) 
#         draw_points_center_car = (np.asarray([line.plot_left + center_car - left_lane, line.ploty]).T).astype(np.int32) 

#         center_poly_color = (250,70,0) 
#         #cv2.circle(img,(img.shape[1]-50, center_car), 50, (0,255,0), -1)
#         cv2.polylines(img, [draw_points_center_lane], False, color=(center_poly_color), thickness=3)
#         cv2.polylines(img, [draw_points_center_car], False, color=(0,255,0), thickness=3)
    
        return diff * xm_per_pix
        
    
def display_hud_text(img, center, curvature):
    font                   = cv2.FONT_HERSHEY_SIMPLEX
    bottomLeftCornerOfText = (75,50)
    fontScale              = 1
    fontColor              = (255,255,255)
    lineType               = 2

    cv2.putText(img, curvature, 
        bottomLeftCornerOfText, 
        font, 
        fontScale,
        fontColor,
        lineType)
    
    bottomLeftCornerOfText = (75,100)
    
    cv2.putText(img, center, 
        bottomLeftCornerOfText, 
        font, 
        fontScale,
        fontColor,
        lineType)

def create_curvature_text(curvature):
    return  'Curvature: {:.2f}m'.format(curvature)
def create_center_text(center):
    direction = "left"
    if(center > 0):
        direction = 'right'
    
    return 'Center: {:.2f}m to the {}'.format(np.abs(center), direction)

def display_hud(img, ploty, left_fit, right_fit):
    if(left_fit != None and right_fit != None):
        center = calculate_center_diff(img)
        curvature = measure_curvature(ploty, left_fit, right_fit)
        print('Center: {:.2f}m \n Curvature: {}'.format(center, curvature))
        center_text = create_center_text(center)
        curvature_text = create_curvature_text(curvature)
        display_hud_text(img, center_text, curvature_text)

Sliding Window¶

import numpy as np
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import cv2


line = Line()


# Load our image
left_fit = None
right_fit = None


def find_lane_pixels(binary_warped, debug = False):
    # Take a histogram of the bottom half of the image
    histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)
    # Create an output image to draw on and visualize the result
    
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))
    # Find the peak of the left and right halves of the histogram
    # These will be the starting point for the left and right lines
    midpoint = np.int(histogram.shape[0]//2)
    leftx_base = np.argmax(histogram[:midpoint])
    rightx_base = np.argmax(histogram[midpoint:]) + midpoint

    # HYPERPARAMETERS
    # Choose the number of sliding windows
    nwindows = 9
    # Set the width of the windows +/- margin
    margin = 100
    # Set minimum number of pixels found to recenter window
    minpix = 50

    # Set height of windows - based on nwindows above and image shape
    window_height = np.int(binary_warped.shape[0]//nwindows)
    # Identify the x and y positions of all nonzero pixels in the image
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    # Current positions to be updated later for each window in nwindows
    leftx_current = leftx_base
    rightx_current = rightx_base

    # Create empty lists to receive left and right lane pixel indices
    left_lane_inds = []
    right_lane_inds = []
    
    # Step through the windows one by one
    for window in range(nwindows):
        # Identify window boundaries in x and y (and right and left)
        win_y_low = binary_warped.shape[0] - (window+1)*window_height
        win_y_high = binary_warped.shape[0] - window*window_height
        ### TO-DO: Find the four below boundaries of the window ###
        win_xleft_low = leftx_current - margin  
        win_xleft_high = leftx_current + margin
        win_xright_low = rightx_current - margin
        win_xright_high = rightx_current + margin
        
        if(debug):
            red = (250,10,10) 
            green = (10,250,10) 
            blue = (10,10,250) 
            cv2.rectangle(out_img, (10, 10), (100, 50), (blue), -1)
            # Draw the windows on the visualization image
            cv2.rectangle(out_img,(win_xleft_low,win_y_low),
            (win_xleft_high,win_y_high),red, 3) 
            cv2.rectangle(out_img,(win_xright_low,win_y_low),
            (win_xright_high,win_y_high), green, 3) 

        


        # Identify the nonzero pixels in x and y within the window #
        good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xleft_low) &  (nonzerox < win_xleft_high)).nonzero()[0]
        good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & 
        (nonzerox >= win_xright_low) &  (nonzerox < win_xright_high)).nonzero()[0]

        # Append these indices to the lists
        left_lane_inds.append(good_left_inds)
        right_lane_inds.append(good_right_inds)
        
        # If you found > minpix pixels, recenter next window ###
        ### (`right` or `leftx_current`) on their mean position ###
        if(len(good_left_inds) > minpix):
            leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
        if(len(good_right_inds) > minpix):
            rightx_current = np.int(np.mean(nonzerox[good_right_inds]))


    # Concatenate the arrays of indices (previously was a list of lists of pixels)
    try:
        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)
    except ValueError:
        # Avoids an error if the above is not implemented fully
        pass

    # Extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]


    return leftx, lefty, rightx, righty, out_img


def fit_polynomial(binary_warped, debug = False):
    
    # Find our lane pixels first
    leftx, lefty, rightx, righty, out_img = find_lane_pixels(binary_warped, debug = debug)
    
    #Fit a second order polynomial to each using `np.polyfit` ###
    print(np.shape(leftx))
    if(np.shape(leftx)[0] == 0 or np.shape(rightx)[0] == 0):
        print('Oh no didnt find any lane pixels')
        return None, None, None, np.dstack((binary_warped, binary_warped, binary_warped))
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)

    # Generate x and y values for plotting
    ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
    try:
        left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
        right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    except TypeError:
        # Avoids an error if `left` and `right_fit` are still none or incorrect
        print('The function failed to fit a line!')
        left_fitx = 1*ploty**2 + 1*ploty
        right_fitx = 1*ploty**2 + 1*ploty

    ## Visualization ##
    # Colors in the left and right lane regions
    blue = (10,10,250) 
    left_poly_color = (250,70,0)
    right_poly_color = (70,250,10)
    left_pixels_color = (220,200,90)
    right_pixels_color = (70,100,200)

    out_img[lefty, leftx] = left_pixels_color
    out_img[righty, rightx] = right_pixels_color
    
    draw_points_left = (np.asarray([left_fitx, ploty]).T).astype(np.int32) 
    draw_points_right = (np.asarray([right_fitx, ploty]).T).astype(np.int32) 

                
    cv2.polylines(out_img, [draw_points_left], False, color=(left_poly_color), thickness=3)
    
    cv2.polylines(out_img, [draw_points_right], False, color=(right_poly_color), thickness=3)
    
    # Plots the left and right polynomials on the lane lines
    #plt.plot(left_fitx, ploty, color='yellow')
    #plt.plot(right_fitx, ploty, color='cyan')
    #plt.imshow(out_img)
    line.plot_left = left_fitx
    line.plot_right = right_fitx
    line.ploty = ploty

    return ploty, left_fit, right_fit, out_img


# Test on the test images.
for file in files:
    if file[0:6] != "output":
        img = mpimg.imread("test_images/" + file)
        
        # Undistort the camera image using the mtx calculated from the chess board.
        undist = cv2.undistort(img, mtx, dist, None, mtx)
        # Transform image into a top down view.
        top_down, m, m_inverse = transform_lane_perspective(undist)
        # Use sob with saturation.
        color_sob = color_sob_pipeline(top_down)
        result = color_sob
        ploty, left_fit, right_fit, result = fit_polynomial(color_sob, debug = True)
        
        # Add Hud Text
        display_hud(img, ploty, left_fit, right_fit)   
            
                               
        histogram = hist(result)
        fig=plt.figure(figsize=(16, 8))
        rows = 2
        cols = 2
        fig.add_subplot(rows, cols, 1)
        plt.imshow(img) 
        fig.add_subplot(rows, cols, 2)
        plt.imshow(result)
        fig.add_subplot(rows, cols, 4)
        plt.title("Histogram of Lines")
        plt.plot(histogram, color = 'red')
        r,g,b =  cv2.split(result)
        result = cv2.merge((b,g,r))
        cv2.imwrite("output_images/output_curve_window_"+file, result)

(9282,)
Center: -0.02m 
 Curvature: 9793.516799960893

/root/miniconda3/envs/carnd-term1/lib/python3.5/site-packages/ipykernel_launcher.py:118: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.

(12755,)
Center: 0.14m 
 Curvature: 1410.9816919760594
(3238,)
Center: 0.21m 
 Curvature: 327.6188676908297
(3462,)
Center: -0.47m 
 Curvature: 14294.329342467823
(17456,)
Center: 0.31m 
 Curvature: 712.0271989475611
(14605,)
Center: -0.05m 
 Curvature: 22242.891972023186
(4168,)
Center: -0.04m 
 Curvature: 853.9152527384675
(21317,)
Center: 0.25m 
 Curvature: 388.5889427407934
(5556,)
Center: 0.38m 
 Curvature: 881.4926585254752

Searching for Line based on last curve position¶

def fit_poly(img_shape, leftx, lefty, rightx, righty):
    # Fit a second order polynomial to each with np.polyfit() ###
    left_fit = np.polyfit(lefty, leftx, 2)
    right_fit = np.polyfit(righty, rightx, 2)
    # Generate x and y values for plotting
    ploty = np.linspace(0, img_shape[0]-1, img_shape[0])
    # Calc both polynomials using ploty, left_fit and right_fit ###
    left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
    right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
    line.plot_left = left_fitx
    line.plot_right = right_fitx
    line.ploty = ploty
    line.current_left_fit = left_fit
    line.current_right_fit = right_fit
    return left_fitx, right_fitx, ploty

def search_around_poly(binary_warped, left_fit, right_fit):
    # HYPERPARAMETER
    # Choose the width of the margin around the previous polynomial to search
    # The quiz grader expects 100 here, but feel free to tune on your own!
    margin = 100

    # Grab activated pixels
    nonzero = binary_warped.nonzero()
    nonzeroy = np.array(nonzero[0])
    nonzerox = np.array(nonzero[1])
    
    ### TO-DO: Set the area of search based on activated x-values ###
    ### within the +/- margin of our polynomial function ###
    ### Hint: consider the window areas for the similarly named variables ###
    ### in the previous quiz, but change the windows to our new search area ###
    left_xs = left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2]
    right_xs = right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2]
    left_lane_inds = ((nonzerox > (left_xs - margin)) & (nonzerox < (left_xs + margin)))
    right_lane_inds = ((nonzerox > (right_xs - margin)) & (nonzerox < (right_xs + margin)))
    
    # Again, extract left and right line pixel positions
    leftx = nonzerox[left_lane_inds]
    lefty = nonzeroy[left_lane_inds] 
    rightx = nonzerox[right_lane_inds]
    righty = nonzeroy[right_lane_inds]

    # Fit new polynomials
    left_fitx, right_fitx, ploty = fit_poly(binary_warped.shape, leftx, lefty, rightx, righty)
    
    ## Visualization ##
    # Create an image to draw on and an image to show the selection window
    out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
    window_img = np.zeros_like(out_img)
    # Color in left and right line pixels
    out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
    out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]

    # Generate a polygon to illustrate the search window area
    # And recast the x and y points into usable format for cv2.fillPoly()
    left_line_window1 = np.array([np.transpose(np.vstack([left_fitx-margin, ploty]))])
    left_line_window2 = np.array([np.flipud(np.transpose(np.vstack([left_fitx+margin, 
                              ploty])))])
    left_line_pts = np.hstack((left_line_window1, left_line_window2))
    right_line_window1 = np.array([np.transpose(np.vstack([right_fitx-margin, ploty]))])
    right_line_window2 = np.array([np.flipud(np.transpose(np.vstack([right_fitx+margin, 
                              ploty])))])
    right_line_pts = np.hstack((right_line_window1, right_line_window2))

    # Draw the lane onto the warped blank image
    cv2.fillPoly(window_img, np.int_([left_line_pts]), (0,255, 0))
    cv2.fillPoly(window_img, np.int_([right_line_pts]), (0,255, 0))
    result = cv2.addWeighted(out_img, 1, window_img, 0.3, 0)
    
    # Plot the polynomial lines onto the image
    plt.plot(left_fitx, ploty, color='yellow')
    plt.plot(right_fitx, ploty, color='yellow')
    ## End visualization steps ##

    return ploty, left_fitx, right_fitx, out_img

def fast_fit_poly(img, debug = False):
    print(line.current_left_fit)
    if(len(line.current_left_fit) == 0 or len(line.current_right_fit) == 0):
        # Calculate fresh.
        print('fresh')
        ploty, left_fit, right_fit, out_img = fit_polynomial(img, debug)
        line.current_left_fit = left_fit
        line.current_right_fit = right_fit
        return ploty, left_fit, right_fit, out_img
        
    else:
        # Search around the last frame. 
        print('fast')
        ploty, left_fit, right_fit, out_img =  search_around_poly(img,
                                                                  line.current_left_fit, 
                                                                  line.current_right_fit)

        return ploty, left_fit, right_fit, out_img
        
# Run image through the pipeline
# Note that in your project, you'll also want to feed in the previous fits
line = Line()
image = mpimg.imread('test_images/test1.jpg')

# Undistort the camera image using the mtx calculated from the chess board.
# Undistort the camera image using the mtx calculated from the chess board.
undist = cv2.undistort(image, mtx, dist, None, mtx)
# Transform image into a top down view.
top_down, m, m_inverse = transform_lane_perspective(undist)
# Use sob with saturation.
color_sob = color_sob_pipeline(top_down)
result = color_sob

ploty, left_fit, right_fit, result = fast_fit_poly(color_sob, debug = True)
curvature_text = 'Curvature: {} '.format(measure_curvature(ploty, left_fit, right_fit))
display_hud(image, ploty, left_fit, right_fit) 
print(curvature_text)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(result)
ax2.set_title('Undistorted Image', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)    
  

# View your output
plt.imshow(result)

r,g,b =  cv2.split(result)
result = cv2.merge((b,g,r))
cv2.imwrite("output_images/output_curve_window_"+file, result)

[]
fresh
(3238,)
Center: 0.21m 
 Curvature: 327.6188676908297
Curvature: 327.6188676908297

/root/miniconda3/envs/carnd-term1/lib/python3.5/site-packages/ipykernel_launcher.py:118: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.

True

Project back¶

def project_back(img, lane, inverse_matrix, debug = False):
    r,g,b =  cv2.split(lane)
    lane = cv2.merge((b,g,r))
    lane_size = (lane.shape[1], lane.shape[0])
    background = np.asarray(img, np.float64())
    if(debug):
        # Warp the images back
        lane_image_space = cv2.warpPerspective(lane, inverse_matrix, lane_size, flags=cv2.INTER_LINEAR)

        lane_image_space = np.asarray(lane_image_space, np.float64())
        overlay = cv2.addWeighted(background,1,lane_image_space,0.7,0,  dtype=cv2.CV_8UC1)

    else:
        mask = np.zeros_like(lane)
        print(np.shape(line.plot_left))
        print(np.shape(line.plot_right))
        print(np.shape(line.ploty))
        left_points = np.array([np.transpose(np.vstack([line.plot_left, line.ploty]))])
        right_points = np.array([np.flipud(np.transpose(np.vstack([line.plot_right, line.ploty])))])
        points = np.hstack((left_points, right_points))
        cv2.fillPoly(mask, np.int_([points]), (0, 255, 0))
        mask_image_space = cv2.warpPerspective(mask, inverse_matrix, lane_size, flags=cv2.INTER_LINEAR)
        overlay = cv2.addWeighted(background,1,mask_image_space,0.7,0,  dtype=cv2.CV_8UC1)


    return overlay

overlay = project_back(image, result, m_inverse)
plt.imshow(overlay)

(720,)
(720,)
(720,)

<matplotlib.image.AxesImage at 0x7f1fa56485c0>

The Pipeline¶

def process_pipeline(image):
    # Undistort the camera image using the mtx calculated from the chess board.
    undist = cv2.undistort(image, mtx, dist, None, mtx)
    # Transform image into a top down view.
    top_down, m, m_inverse = transform_lane_perspective(undist)
    # Use sob with saturation.
    binary = color_sob_pipeline(top_down)
    #curves = fit_polynomial(binary)
    ploty, left_fit, right_fit, curves = fast_fit_poly(binary, debug = True)
    overlay = project_back(image, curves, m_inverse)
    curvature_text = 'Curvature: {} '.format(measure_curvature(ploty, left_fit, right_fit))
    display_hud(overlay, ploty, left_fit, right_fit) 
    print(curvature_text)
    result = overlay
    return result

Run the pipeline¶

# Test on the test images.
for file in files:
    if file[0:6] != "output":
        img = mpimg.imread("test_images/" + file)
        line = Line()
        result = process_pipeline(img)
        fig=plt.figure(figsize=(16, 8))
        rows = 1
        cols = 2
        fig.add_subplot(rows, cols, 1)
        plt.imshow(img) 
        fig.add_subplot(rows, cols, 2)
        plt.title(file)
        plt.imshow(result)
        
        r,g,b =  cv2.split(result)
        result = cv2.merge((b,g,r))
        cv2.imwrite("output_images/output_pipeline_"+file, result)

[]
fresh
(9282,)
(720,)
(720,)
(720,)
Center: -0.02m 
 Curvature: 9793.516799960893
Curvature: 9793.516799960893

/root/miniconda3/envs/carnd-term1/lib/python3.5/site-packages/ipykernel_launcher.py:118: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.

[]
fresh
(12755,)
(720,)
(720,)
(720,)
Center: 0.14m 
 Curvature: 1410.9816919760594
Curvature: 1410.9816919760594 
[]
fresh
(3238,)
(720,)
(720,)
(720,)
Center: 0.21m 
 Curvature: 327.6188676908297
Curvature: 327.6188676908297 
[]
fresh
(3462,)
(720,)
(720,)
(720,)
Center: -0.47m 
 Curvature: 14294.329342467823
Curvature: 14294.329342467823 
[]
fresh
(17456,)
(720,)
(720,)
(720,)
Center: 0.31m 
 Curvature: 712.0271989475611
Curvature: 712.0271989475611 
[]
fresh
(14605,)
(720,)
(720,)
(720,)
Center: -0.05m 
 Curvature: 22242.891972023186
Curvature: 22242.891972023186 
[]
fresh
(4168,)
(720,)
(720,)
(720,)
Center: -0.04m 
 Curvature: 853.9152527384675
Curvature: 853.9152527384675 
[]
fresh
(21317,)
(720,)
(720,)
(720,)
Center: 0.25m 
 Curvature: 388.5889427407934
Curvature: 388.5889427407934 
[]
fresh
(5556,)
(720,)
(720,)
(720,)
Center: 0.38m 
 Curvature: 881.4926585254752
Curvature: 881.4926585254752

Test on Video¶

# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML

Imageio: 'ffmpeg.linux64' was not found on your computer; downloading it now.
Try 1. Download from https://github.com/imageio/imageio-binaries/raw/master/ffmpeg/ffmpeg.linux64 (27.2 MB)
Downloading: 8192/28549024 bytes (0.0122880/28549024 bytes (0.4253952/28549024 bytes (0.9417792/28549024 bytes (1.5548864/28549024 bytes (1.9630784/28549024 bytes (2.2679936/28549024 bytes (2.4745472/28549024 bytes (2.6811008/28549024 bytes (2.8876544/28549024 bytes (3.1925696/28549024 bytes (3.2991232/28549024 bytes (3.51056768/28549024 bytes (3.7%1122304/28549024 bytes (3.9%1187840/28549024 bytes (4.2%1269760/28549024 bytes (4.4%1351680/28549024 bytes (4.7%1433600/28549024 bytes (5.0%1531904/28549024 bytes (5.4%1613824/28549024 bytes (5.7%1646592/28549024 bytes (5.8%1777664/28549024 bytes (6.2%1875968/28549024 bytes (6.6%1941504/28549024 bytes (6.8%2023424/28549024 bytes (7.1%2056192/28549024 bytes (7.2%2154496/28549024 bytes (7.5%2269184/28549024 bytes (7.9%2424832/28549024 bytes (8.5%2580480/28549024 bytes (9.0%2744320/28549024 bytes (9.6%2908160/28549024 bytes (10.23072000/28549024 bytes (10.83235840/28549024 bytes (11.33416064/28549024 bytes (12.03497984/28549024 bytes (12.33629056/28549024 bytes (12.73792896/28549024 bytes (13.33956736/28549024 bytes (13.94120576/28549024 bytes (14.44284416/28549024 bytes (15.04448256/28549024 bytes (15.64620288/28549024 bytes (16.24775936/28549024 bytes (16.74939776/28549024 bytes (17.35103616/28549024 bytes (17.95251072/28549024 bytes (18.45414912/28549024 bytes (19.05578752/28549024 bytes (19.55742592/28549024 bytes (20.15890048/28549024 bytes (20.66053888/28549024 bytes (21.26217728/28549024 bytes (21.86365184/28549024 bytes (22.36529024/28549024 bytes (22.96692864/28549024 bytes (23.46823936/28549024 bytes (23.96971392/28549024 bytes (24.47151616/28549024 bytes (25.17315456/28549024 bytes (25.67479296/28549024 bytes (26.27643136/28549024 bytes (26.87708672/28549024 bytes (27.07774208/28549024 bytes (27.27839744/28549024 bytes (27.57987200/28549024 bytes (28.08134656/28549024 bytes (28.58298496/28549024 bytes (29.18462336/28549024 bytes (29.68626176/28549024 bytes (30.28790016/28549024 bytes (30.88953856/28549024 bytes (31.49117696/28549024 bytes (31.99281536/28549024 bytes (32.59461760/28549024 bytes (33.19625600/28549024 bytes (33.79789440/28549024 bytes (34.39936896/28549024 bytes (34.810125312/28549024 bytes (35.5%10280960/28549024 bytes (36.0%10428416/28549024 bytes (36.5%10608640/28549024 bytes (37.2%10772480/28549024 bytes (37.7%10936320/28549024 bytes (38.3%11083776/28549024 bytes (38.8%11264000/28549024 bytes (39.5%11427840/28549024 bytes (40.0%11591680/28549024 bytes (40.6%11722752/28549024 bytes (41.1%11927552/28549024 bytes (41.8%12083200/28549024 bytes (42.3%12247040/28549024 bytes (42.9%12410880/28549024 bytes (43.5%12574720/28549024 bytes (44.0%12738560/28549024 bytes (44.6%12902400/28549024 bytes (45.2%13066240/28549024 bytes (45.8%13230080/28549024 bytes (46.3%13393920/28549024 bytes (46.9%13557760/28549024 bytes (47.5%13721600/28549024 bytes (48.1%13885440/28549024 bytes (48.6%14024704/28549024 bytes (49.1%14180352/28549024 bytes (49.7%14344192/28549024 bytes (50.2%14508032/28549024 bytes (50.8%14671872/28549024 bytes (51.4%14819328/28549024 bytes (51.9%14983168/28549024 bytes (52.5%15065088/28549024 bytes (52.8%15179776/28549024 bytes (53.2%15294464/28549024 bytes (53.6%15376384/28549024 bytes (53.9%15458304/28549024 bytes (54.1%15540224/28549024 bytes (54.4%15704064/28549024 bytes (55.0%15867904/28549024 bytes (55.6%16031744/28549024 bytes (56.2%16195584/28549024 bytes (56.7%16359424/28549024 bytes (57.3%16523264/28549024 bytes (57.9%16687104/28549024 bytes (58.5%16850944/28549024 bytes (59.0%17014784/28549024 bytes (59.6%17178624/28549024 bytes (60.2%17342464/28549024 bytes (60.7%17506304/28549024 bytes (61.3%17686528/28549024 bytes (62.0%17850368/28549024 bytes (62.5%18014208/28549024 bytes (63.1%18161664/28549024 bytes (63.6%18325504/28549024 bytes (64.2%18505728/28549024 bytes (64.8%18669568/28549024 bytes (65.4%18833408/28549024 bytes (66.0%18997248/28549024 bytes (66.5%19161088/28549024 bytes (67.1%19324928/28549024 bytes (67.7%19488768/28549024 bytes (68.3%19652608/28549024 bytes (68.8%19767296/28549024 bytes (69.2%19898368/28549024 bytes (69.7%19947520/28549024 bytes (69.9%20193280/28549024 bytes (70.7%20275200/28549024 bytes (71.0%20357120/28549024 bytes (71.3%20439040/28549024 bytes (71.6%20520960/28549024 bytes (71.9%20701184/28549024 bytes (72.5%20799488/28549024 bytes (72.9%20865024/28549024 bytes (73.1%20946944/28549024 bytes (73.4%21028864/28549024 bytes (73.7%21094400/28549024 bytes (73.9%21110784/28549024 bytes (73.9%21159936/28549024 bytes (74.1%21241856/28549024 bytes (74.4%21323776/28549024 bytes (74.7%21405696/28549024 bytes (75.0%21454848/28549024 bytes (75.2%21471232/28549024 bytes (75.2%21700608/28549024 bytes (76.0%21864448/28549024 bytes (76.6%22028288/28549024 bytes (77.2%22192128/28549024 bytes (77.7%22355968/28549024 bytes (78.3%22470656/28549024 bytes (78.7%22634496/28549024 bytes (79.3%22798336/28549024 bytes (79.9%22962176/28549024 bytes (80.4%23126016/28549024 bytes (81.0%23273472/28549024 bytes (81.5%23437312/28549024 bytes (82.1%23601152/28549024 bytes (82.7%23764992/28549024 bytes (83.2%23928832/28549024 bytes (83.8%24092672/28549024 bytes (84.4%24256512/28549024 bytes (85.0%24420352/28549024 bytes (85.5%24584192/28549024 bytes (86.1%24748032/28549024 bytes (86.7%24911872/28549024 bytes (87.3%25075712/28549024 bytes (87.8%25239552/28549024 bytes (88.4%25403392/28549024 bytes (89.0%25567232/28549024 bytes (89.6%25714688/28549024 bytes (90.1%25894912/28549024 bytes (90.7%26058752/28549024 bytes (91.3%26222592/28549024 bytes (91.9%26386432/28549024 bytes (92.4%26550272/28549024 bytes (93.0%26714112/28549024 bytes (93.6%26877952/28549024 bytes (94.1%27041792/28549024 bytes (94.7%27205632/28549024 bytes (95.3%27385856/28549024 bytes (95.9%27484160/28549024 bytes (96.3%27582464/28549024 bytes (96.6%27664384/28549024 bytes (96.9%27762688/28549024 bytes (97.2%27893760/28549024 bytes (97.7%28008448/28549024 bytes (98.1%28106752/28549024 bytes (98.5%28270592/28549024 bytes (99.0%28434432/28549024 bytes (99.6%28549024/28549024 bytes (100.0%)
  Done
File saved as /root/.imageio/ffmpeg/ffmpeg.linux64.

def process_image(image):
    # NOTE: The output you return should be a color image (3 channel) for processing video below
    # TODO: put your pipeline here,
    # you should return the final output (image where lines are drawn on lanes)
    result  = process_pipeline(image)
    return result

First Video¶

line = Line()
project_output = 'test_videos_output/solidWhiteRight.mp4'
left_fit = None
right_fit = None

## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("solidWhiteRight.mp4")
project_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time project_clip.write_videofile(project_output, audio=False)

[]
fresh
(5976,)
(540,)
(540,)
(540,)
Center: -0.01m 
 Curvature: 969.6141630695898
Curvature: 969.6141630695898 
[MoviePy] >>>> Building video test_videos_output/solidWhiteRight.mp4
[MoviePy] Writing video test_videos_output/solidWhiteRight.mp4

  0%|          | 1/222 [00:00<01:21,  2.70it/s]

[ -2.45264668e-04   3.08753678e-01   1.26156988e+02]
fast
(540,)
(540,)
(540,)
Center: 0.00m 
 Curvature: 105.0966784807158
Curvature: 105.0966784807158

  1%|          | 2/222 [00:00<01:14,  2.94it/s]

[ -1.46792891e-04   2.68220824e-01   1.28835765e+02]
fast
(540,)
(540,)
(540,)
Center: 0.00m 
 Curvature: 104.39084214976332
Curvature: 104.39084214976332

  1%|▏         | 3/222 [00:00<01:09,  3.14it/s]

[ -1.61821895e-04   2.75751327e-01   1.30791539e+02]
fast
(540,)
(540,)
(540,)
Center: 0.00m 
 Curvature: 104.18973921253946
Curvature: 104.18973921253946

  2%|▏         | 4/222 [00:01<01:05,  3.30it/s]

[ -2.01889255e-04   3.04526474e-01   1.26836712e+02]
fast
(540,)
(540,)
(540,)
Center: -0.01m 
 Curvature: 104.20637251764187
Curvature: 104.20637251764187

  2%|▏         | 5/222 [00:01<01:02,  3.46it/s]

line = Line()
project_output = 'test_videos_output/ouput_project_video.mp4'
left_fit = None
right_fit = None

## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip("project_video.mp4")
project_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time project_clip.write_videofile(project_output, audio=False)

[]
fresh
(22071,)
(720,)
(720,)
(720,)
Center: 0.32m 
 Curvature: 473.9150322961419
Curvature: 473.9150322961419 
[MoviePy] >>>> Building video test_videos_output/ouput_project_video.mp4
[MoviePy] Writing video test_videos_output/ouput_project_video.mp4

  0%|          | 1/1261 [00:00<12:20,  1.70it/s]

[ -4.09577710e-04   5.60398124e-01   1.25231920e+02]
fast
(720,)
(720,)
(720,)
Center: 0.33m 
 Curvature: 119.22066951830142
Curvature: 119.22066951830142

  0%|          | 2/1261 [00:01<11:33,  1.82it/s]

[ -4.07802096e-04   5.59547946e-01   1.25294910e+02]
fast
(720,)
(720,)
(720,)
Center: 0.33m 
 Curvature: 116.81155102446287
Curvature: 116.81155102446287

  0%|          | 3/1261 [00:01<10:53,  1.92it/s]

[ -4.22361879e-04   5.72610931e-01   1.23863307e+02]
fast
(720,)
(720,)
(720,)
Center: 0.33m 
 Curvature: 114.88435598316784
Curvature: 114.88435598316784

  0%|          | 4/1261 [00:01<10:21,  2.02it/s]

[ -4.32968614e-04   5.83798702e-01   1.21832851e+02]
fast
(720,)
(720,)
(720,)
Center: 0.33m 
 Curvature: 112.18437473392184
Curvature: 112.18437473392184

  0%|          | 5/1261 [00:02<09:57,  2.10it/s]

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(project_output))

Challenge Video Video¶

line = Line()
challenge_output = 'test_videos_output/ouput_challenge_video.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip2 = VideoFileClip("challenge_video.mp4")
challenge_clip = clip2.fl_image(process_image) #NOTE: this function expects color images!!
%time challenge_clip.write_videofile(challenge_output, audio=False)

[]
fresh
(3538,)
(720,)
(720,)
(720,)
Center: 0.16m 
 Curvature: 290.82919192868565
Curvature: 290.82919192868565 
[MoviePy] >>>> Building video test_videos_output/ouput_challenge_video.mp4
[MoviePy] Writing video test_videos_output/ouput_challenge_video.mp4

  0%|          | 1/485 [00:00<04:43,  1.71it/s]

[  6.16336473e-04  -1.37189498e+00   9.93211101e+02]
fast
(720,)
(720,)
(720,)
Center: 0.16m 
 Curvature: 369.6208927019936
Curvature: 369.6208927019936

  0%|          | 2/485 [00:01<04:22,  1.84it/s]

[  6.16336473e-04  -1.37189498e+00   9.93211101e+02]
fast
(720,)
(720,)
(720,)
Center: 0.17m 
 Curvature: 403.18516316027626
Curvature: 403.18516316027626

  1%|          | 3/485 [00:01<04:04,  1.97it/s]

[  7.40885677e-04  -1.54365506e+00   1.05152054e+03]
fast
(720,)
(720,)
(720,)
Center: 0.17m 
 Curvature: 408.00777039006545
Curvature: 408.00777039006545

  1%|          | 4/485 [00:01<03:51,  2.08it/s]

[  7.13279869e-04  -1.52072141e+00   1.04951336e+03]
fast
(720,)
(720,)
(720,)
Center: 0.16m 
 Curvature: 418.7792022291331
Curvature: 418.7792022291331

  1%|          | 5/485 [00:02<03:42,  2.16it/s]

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(challenge_output))

Harder Challenge Video Video¶

harder_challenge_output = 'test_videos_output/ouput_harder_challenge_video.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip2 = VideoFileClip("harder_challenge_video.mp4")
harder_challenge_clip = clip2.fl_image(process_image) #NOTE: this function expects color images!!
%time harder_challenge_clip.write_videofile(harder_challenge_output, audio=False)

[  5.56541037e-03  -5.17845935e+00   1.26496062e+03]
fast
(720,)
(720,)
(720,)
Center: 0.47m 
 Curvature: 1260.5841086412381
Curvature: 1260.5841086412381 
[MoviePy] >>>> Building video test_videos_output/ouput_harder_challenge_video.mp4
[MoviePy] Writing video test_videos_output/ouput_harder_challenge_video.mp4

  0%|          | 1/1200 [00:00<10:04,  1.98it/s]

[  1.59134440e-04   1.85848889e-01   5.02329480e+01]
fast
(720,)
(720,)
(720,)
Center: 0.37m 
 Curvature: 1292.3954197048777
Curvature: 1292.3954197048777

  0%|          | 2/1200 [00:00<09:41,  2.06it/s]

[ -8.49807600e-04   1.19481156e+00  -1.81744607e+02]
fast
(720,)
(720,)
(720,)
Center: 0.38m 
 Curvature: 1172.6663599062829
Curvature: 1172.6663599062829

  0%|          | 3/1200 [00:01<09:17,  2.15it/s]

[ -8.07971579e-04   1.17110196e+00  -1.89522874e+02]
fast
(720,)
(720,)
(720,)
Center: 0.37m 
 Curvature: 1117.3584724715645
Curvature: 1117.3584724715645

  0%|          | 4/1200 [00:01<08:59,  2.22it/s]

[ -7.25401516e-04   1.09849421e+00  -1.79980204e+02]
fast
(720,)
(720,)
(720,)
Center: 0.34m 
 Curvature: 1154.712315531683
Curvature: 1154.712315531683

  0%|          | 5/1200 [00:02<08:46,  2.27it/s]

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(harder_challenge_output))

After¶

def warp(img):
    img_size = (img.shape[1], img.shape[0])
    # Source src co-ords
    src = np.float32(
        [[850, 320],
         [865, 450],
         [533, 350],
         [535, 210]]
    )
    # Four desired coordinates
    dst = np.float32(
        [[870, 240],
         [870, 370],
         [520, 370],
         [520, 240]]
    )

    # Compute the perspective transform M
    m = cv2.getPerspectiveTransform(src, dst)
    # inverse by swapping
    
    # Warp the images
    warped = cv2.warpPerspective(img, m, img_size, flags=cv2.INTER_LINEAR)
    return warped